# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
from scrapy.exceptions import DropItem
from storeDB import NewsDB

class TutorialPipeline(object):
    """A pipeline for filtering out items which contain certain words in their
    description"""

    # put all words in lowercase
    words_to_filter = ['politics', 'religion']

    # def process_item(self, item, spider):
    #     for word in self.words_to_filter:
    #         if word in unicode(item['description']).lower():
    #             raise DropItem("Contains forbidden word: %s" % word)
    #     else:
    #         return item

    def process_item(self, item, spider):
        if spider.name != "news":  return item
        if item.get("news_thread", None) is None: return item

        spec = { "news_thread": item["news_thread"] }
        NewsDB.test414.update(spec, {'$set': dict(item)}, upsert=True)

        return None